CI / test (push) Successful in 1m38s
The accounting layer has always exempted a model that costs nothing, with a comment saying it is "what lets a key with no budget left still be served a cheap fallback". That described a caller nobody had written: the budget was checked once, against the requested model, and a refusal was the end of it, so the exemption could never fire. A caller that has run out is now offered the first model in its fallbacks it can still afford. The allowlist still applies — running out of money is not a way to reach a model a key was never allowed — and a model with no fallback is refused exactly as before, or a budget would stop meaning anything. Only a spent budget falls back: a rate limit is answered by waiting a second, and moving the request elsewhere would spend someone else's capacity to avoid a delay nobody asked to avoid. Which uncovered a header that has never worked. x-llmgw-served-model was set in finish, and finish runs after the proxy has written the response — so the one mechanism for telling a client that something else answered has been announcing it to nobody since it was written. Nothing failed, because the only thing asserting it was a comment. It is now set before dispatch, for the capacity fallback as well as this one. Finding that took bisecting a test I had written wrong: it polled for a 429 that never came, because the budget had run out and the fallback had served the request — the feature working, reported as the feature failing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
320 lines
9.6 KiB
Go
320 lines
9.6 KiB
Go
package e2e
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// paidConfig turns on auth with a price attached to the test model, so spend
|
|
// is a number rather than always zero.
|
|
func paidConfig(path string) func(*config.Server) {
|
|
return func(c *config.Server) {
|
|
c.Auth = config.Auth{
|
|
Enabled: true, Store: path,
|
|
CacheTTL: 50 * time.Millisecond,
|
|
FlushInterval: 20 * time.Millisecond,
|
|
BatchSize: 8,
|
|
Pricing: map[string]config.Price{
|
|
testModel: {PromptPer1k: 1_000_000, CompletionPer1k: 1_000_000},
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
func usageRows(t *testing.T, path string) []store.Usage {
|
|
t.Helper()
|
|
s, err := store.OpenSQLite(context.Background(), path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
|
|
rows, err := s.SpendSince(context.Background(), time.Now().Add(-time.Hour))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out := make([]store.Usage, 0, len(rows))
|
|
for key, micros := range rows {
|
|
out = append(out, store.Usage{KeyID: key, CostMicros: micros})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestUsageReachesTheStoreWithoutBlockingRequests(t *testing.T) {
|
|
path, mint := provision(t)
|
|
key := mint(t, "spender")
|
|
h := start(t, 2, paidConfig(path))
|
|
|
|
for range 12 {
|
|
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
|
t.Fatalf("%d %s", code, msg)
|
|
}
|
|
}
|
|
|
|
// The writer is asynchronous, so this is the one place a wait is correct.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 {
|
|
return
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
t.Fatal("no usage reached the store")
|
|
}
|
|
|
|
func TestRequestRateLimitsBindThroughTheGateway(t *testing.T) {
|
|
path, mint := provision(t)
|
|
key := mint(t, "burst")
|
|
withStore(t, path, func(s *store.SQLite) {
|
|
rpm := 5
|
|
if err := s.PutLimits(context.Background(), store.Limits{
|
|
Scope: store.ScopeKey, ScopeID: "key-burst", RPM: &rpm,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
h := start(t, 2, paidConfig(path))
|
|
|
|
ok, limited := 0, 0
|
|
for range 20 {
|
|
switch code, _ := h.call(t, key, testModel); code {
|
|
case http.StatusOK:
|
|
ok++
|
|
case http.StatusTooManyRequests:
|
|
limited++
|
|
default:
|
|
t.Fatalf("unexpected status %d", code)
|
|
}
|
|
}
|
|
if ok != 5 {
|
|
t.Errorf("a limit of 5 let %d requests through", ok)
|
|
}
|
|
if limited != 15 {
|
|
t.Errorf("%d requests were refused, expected 15", limited)
|
|
}
|
|
}
|
|
|
|
func TestASpentBudgetRefusesAndDoesNotSuggestRetrying(t *testing.T) {
|
|
path, mint := provision(t)
|
|
key := mint(t, "broke")
|
|
withStore(t, path, func(s *store.SQLite) {
|
|
// One micro-unit: the first request exhausts it.
|
|
budget := int64(1)
|
|
if err := s.PutLimits(context.Background(), store.Limits{
|
|
Scope: store.ScopeKey, ScopeID: "key-broke",
|
|
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
h := start(t, 2, paidConfig(path))
|
|
|
|
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
|
t.Fatalf("the first request should go through: %d %s", code, msg)
|
|
}
|
|
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests {
|
|
return
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
t.Fatal("a key kept spending past its budget")
|
|
}
|
|
|
|
func TestBudgetSurvivesAGatewayRestart(t *testing.T) {
|
|
// The counters live in memory. If a restart reset them, a budget would be a
|
|
// suggestion that anyone could clear by bouncing the process.
|
|
path, mint := provision(t)
|
|
key := mint(t, "persistent")
|
|
|
|
h := start(t, 2, paidConfig(path))
|
|
for range 3 {
|
|
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
|
t.Fatalf("%d %s", code, msg)
|
|
}
|
|
}
|
|
// Let the batch writer land before the process goes away.
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if rows := usageRows(t, path); len(rows) > 0 && rows[0].CostMicros > 0 {
|
|
break
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
|
|
spent := usageRows(t, path)
|
|
if len(spent) == 0 || spent[0].CostMicros == 0 {
|
|
t.Fatal("nothing was recorded to restore from")
|
|
}
|
|
|
|
// A second gateway over the same store, with a budget just under what was
|
|
// already spent: it must refuse immediately rather than starting fresh.
|
|
withStore(t, path, func(s *store.SQLite) {
|
|
budget := spent[0].CostMicros
|
|
if err := s.PutLimits(context.Background(), store.Limits{
|
|
Scope: store.ScopeKey, ScopeID: "key-persistent",
|
|
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
h2 := start(t, 2, paidConfig(path))
|
|
if code, _ := h2.call(t, key, testModel); code != http.StatusTooManyRequests {
|
|
t.Errorf("the restarted gateway gave the key a fresh budget: %d", code)
|
|
}
|
|
}
|
|
|
|
func TestLimitsAreNotAppliedWithoutAuthentication(t *testing.T) {
|
|
// Accounting only exists where there is an identity to charge.
|
|
h := start(t, 2)
|
|
for range 30 {
|
|
if code, msg := h.call(t, "", testModel); code != http.StatusOK {
|
|
t.Fatalf("an unauthenticated gateway rate-limited a request: %d %s", code, msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
const cheapModel = "cheap-model"
|
|
|
|
// budgetFallbackConfig prices the main model and leaves the fallback free,
|
|
// which is what makes it affordable to a key with nothing left.
|
|
func budgetFallbackConfig(path string) func(*config.Server) {
|
|
return func(c *config.Server) {
|
|
paidConfig(path)(c)
|
|
// The fallback has no price, so it costs nothing and the budget check
|
|
// exempts it. That exemption has existed all along with a comment
|
|
// saying it is what lets a spent key still be served something cheap;
|
|
// nothing ever tried, so it could never fire.
|
|
c.Models = budgetFallbackModels()
|
|
// The same engines answer for both names, so this is about accounting
|
|
// rather than about placement.
|
|
for i := range c.Endpoints {
|
|
c.Endpoints[i].Models = []string{testModel, cheapModel}
|
|
}
|
|
}
|
|
}
|
|
|
|
// callWithHeaders is call, keeping the response headers.
|
|
func (h *harness) callWithHeaders(t *testing.T, key, model string) (int, http.Header) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(map[string]any{
|
|
"model": model,
|
|
"messages": []map[string]string{{"role": "user", "content": "hi"}},
|
|
"max_tokens": 4,
|
|
})
|
|
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost,
|
|
h.router.URL+"/v1/chat/completions", bytes.NewReader(body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+key)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer resp.Body.Close()
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
return resp.StatusCode, resp.Header
|
|
}
|
|
|
|
func TestASpentBudgetIsServedAnAffordableModel(t *testing.T) {
|
|
// A team trying models will exhaust a quota, and being dropped to a smaller
|
|
// model is a far better answer than a refusal: it makes a budget a
|
|
// guardrail rather than a wall.
|
|
path, mint := provision(t)
|
|
key := mint(t, "broke")
|
|
withStore(t, path, func(s *store.SQLite) {
|
|
budget := int64(1) // one micro-unit: the first request exhausts it
|
|
if err := s.PutLimits(context.Background(), store.Limits{
|
|
Scope: store.ScopeKey, ScopeID: "key-broke",
|
|
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
h := start(t, 2, budgetFallbackConfig(path))
|
|
|
|
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
|
t.Fatalf("the first request should go through: %d %s", code, msg)
|
|
}
|
|
|
|
// Once the spend lands, the priced model is unaffordable and the free one
|
|
// answers instead — with the response saying which, rather than leaving
|
|
// the caller to infer it from the reply.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
code, hdr := h.callWithHeaders(t, key, testModel)
|
|
if code == http.StatusOK && hdr.Get("x-llmgw-served-model") == cheapModel {
|
|
return
|
|
}
|
|
if code == http.StatusTooManyRequests {
|
|
t.Fatal("a spent budget refused the request instead of falling back")
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
t.Fatal("the budget never ran out, or the fallback never served")
|
|
}
|
|
|
|
func TestASpentBudgetWithNoFallbackStillRefuses(t *testing.T) {
|
|
// The limit has to remain a limit. A model with nowhere cheaper to go is
|
|
// refused exactly as before, or a budget would stop meaning anything.
|
|
path, mint := provision(t)
|
|
key := mint(t, "broke")
|
|
withStore(t, path, func(s *store.SQLite) {
|
|
budget := int64(1)
|
|
if err := s.PutLimits(context.Background(), store.Limits{
|
|
Scope: store.ScopeKey, ScopeID: "key-broke",
|
|
Budget: &budget, BudgetPeriod: store.PeriodMonth,
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
h := start(t, 2, paidConfig(path)) // no catalog, so no fallbacks
|
|
|
|
if code, msg := h.call(t, key, testModel); code != http.StatusOK {
|
|
t.Fatalf("the first request should go through: %d %s", code, msg)
|
|
}
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if code, _ := h.call(t, key, testModel); code == http.StatusTooManyRequests {
|
|
return
|
|
}
|
|
time.Sleep(25 * time.Millisecond)
|
|
}
|
|
t.Fatal("a key with no fallback kept spending past its budget")
|
|
}
|
|
|
|
// budgetFallbackModels is a priced model with a free one behind it.
|
|
func budgetFallbackModels() []catalog.Model {
|
|
return []catalog.Model{
|
|
{
|
|
Name: testModel, Source: "/models/test",
|
|
Fallbacks: []string{cheapModel},
|
|
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
|
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
|
},
|
|
{
|
|
Name: cheapModel, Source: "/models/cheap",
|
|
Scaling: catalog.Scaling{Min: 1, Max: 1},
|
|
Profiles: []catalog.Profile{{Name: "p", Engine: "llamacpp", Weight: 1}},
|
|
},
|
|
}
|
|
}
|