Files
llmgw/internal/router/pricing_test.go
T
Ilker ManapandClaude Opus 5.5 f8572fb5ee Follow one request from its id to the replica that served it
"Why was this request slow?" had no answer. Nothing tied a request to the
replica that served it, the time admission held it went out in a response
header and was never kept, and the only per-request log lines were for
disconnects, early stream ends and upstream retries.

Every request now has an id: the client's X-Request-Id when it is a plain
string of reasonable length, otherwise one the gateway makes. It is set on
the response before anything else, so a request refused as malformed can be
named too, and forwarded to the engine. One access log line per request
carries it with the model asked for and served, the replica, status, time in
the queue, time to the first token, total time, tokens, and the key,
project and user.

Usage records gain the id, the replica, queue time and time to first token,
tried against a restored copy of the live store before being shipped: 591
rows kept, columns and index in place. `llmgw ctl request ID` reads one back
through the admin API.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 02:36:45 +02:00

191 lines
6.1 KiB
Go

package router
import (
"crypto/sha256"
"path/filepath"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// pricedRouter builds a gateway with authentication, one model under an alias,
// and a price table that charges that model and almost nothing for anything
// else — so a request priced by the wrong name is visible in the figure rather
// than merely plausible.
func pricedRouter(t *testing.T, engineURL string, served ...string) (*Router, string) {
t.Helper()
cfg := baseConfig(config.Endpoint{
ID: "a", URL: engineURL,
Models: append([]string{"qwen2.5-14b-instruct"}, served...),
})
cfg.Models = []catalog.Model{{
Name: "qwen2.5-14b-instruct", Aliases: []string{"q14"},
}}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
cfg.Auth.Pricing = map[string]config.Price{
"qwen2.5-14b-instruct": {PromptPer1k: 14000, CompletionPer1k: 42000},
}
// Deliberately not zero: a fallback of nothing would make a miss look the
// same as a model that is free.
cfg.Auth.DefaultPrice = config.Price{PromptPer1k: 1, CompletionPer1k: 1}
cfg.Auth.FlushInterval = 20 * time.Millisecond
cfg.Auth.BatchSize = 1
rt := newTestRouter(t, cfg)
org, err := rt.store.CreateOrg(t.Context(), "acme")
if err != nil {
t.Fatal(err)
}
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
if err != nil {
t.Fatal(err)
}
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
if err != nil {
t.Fatal(err)
}
// A real key shape: authentication rejects anything else before it looks.
secret := store.KeyPrefix + strings.Repeat("a", 43)
sum := sha256.Sum256([]byte(secret))
if err := rt.store.CreateKey(t.Context(), store.Key{
ID: "key-1", Hash: sum[:], Prefix: "aaaa", Name: "test",
ProjectID: project, CreatedAt: time.Now(),
}); err != nil {
t.Fatal(err)
}
return rt, secret
}
func TestAnAliasIsPricedAsTheModelItResolvesTo(t *testing.T) {
// The price table is written against the catalog, and a caller asks for
// whichever name they were given. Charging by the string they typed meant
// every alias missed the table and fell to the default — so a budget could
// be spent at the wrong rate, or not at all, depending on which name went
// into the request.
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
rec := post(t, rt.Handler(),
map[string]any{"model": "q14", "messages": []any{
map[string]any{"role": "user", "content": "hello"}}},
map[string]string{"Authorization": "Bearer " + secret})
if rec.Code != 200 {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
u := waitForUsage(t, rt)
want := charge(u.Prompt, 14000) + charge(u.Completion, 42000)
fallback := charge(u.Prompt, 1) + charge(u.Completion, 1)
if u.CostMicros != want {
t.Errorf("charged %d micros for %d+%d tokens, expected %d; the "+
"fallback rate would have charged %d",
u.CostMicros, u.Prompt, u.Completion, want, fallback)
}
}
// charge is Price.Cost for one half of a request, rounding up the way the
// accountant does — spelled out here so the expected figure is arithmetic the
// test states rather than the code under test restated.
func charge(tokens, per1k int64) int64 {
if tokens <= 0 {
return 0
}
return (tokens*per1k + 999) / 1000
}
func TestAModelWithNoPriceFallsBackRatherThanCostingNothing(t *testing.T) {
e := newFakeEngine(t, "unpriced")
rt, secret := pricedRouter(t, e.srv.URL, "unpriced")
// Not in the catalog and not in the price table. It must still cost the
// fallback, because a model nobody priced is the one most likely to be new.
rec := post(t, rt.Handler(),
map[string]any{"model": "unpriced", "messages": []any{
map[string]any{"role": "user", "content": "hello"}}},
map[string]string{"Authorization": "Bearer " + secret})
if rec.Code != 200 {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
u := waitForUsage(t, rt)
want := charge(u.Prompt, 1) + charge(u.Completion, 1)
if u.CostMicros != want {
t.Errorf("charged %d micros for %d+%d tokens, expected the fallback's %d",
u.CostMicros, u.Prompt, u.Completion, want)
}
if u.CostMicros == 0 {
t.Error("an unpriced model cost nothing; the fallback did not apply")
}
}
func waitForUsage(t *testing.T, rt *Router) store.KeyUsage {
t.Helper()
since := time.Now().Add(-time.Hour)
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
rows, err := rt.store.UsageSince(t.Context(), since)
if err == nil && len(rows) > 0 {
return rows[0]
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("no usage was recorded")
return store.KeyUsage{}
}
func TestARequestCanBeFollowedByItsID(t *testing.T) {
// "Why was my request slow?" had no answer: the queue time went out in a
// header and nowhere else, and nothing tied a request to its replica.
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
rec := post(t, rt.Handler(),
map[string]any{"model": "q14", "messages": []any{
map[string]any{"role": "user", "content": "hello"}}},
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "client-abc.1"})
if rec.Code != 200 {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if got := rec.Header().Get("X-Request-Id"); got != "client-abc.1" {
t.Errorf("the client's id came back as %q", got)
}
deadline := time.Now().Add(5 * time.Second)
for {
u, ok, err := rt.store.UsageByRequest(t.Context(), "client-abc.1")
if err != nil {
t.Fatal(err)
}
if ok {
if u.Replica == "" || u.Status != 200 {
t.Errorf("recorded %+v; expected the replica that answered", u)
}
break
}
if time.Now().After(deadline) {
t.Fatal("the request's usage cannot be found by its id")
}
time.Sleep(20 * time.Millisecond)
}
}
func TestAnIDThatIsNotOneIsReplaced(t *testing.T) {
for _, bad := range []string{"", "has space", "new\nline", strings.Repeat("x", 200)} {
got := requestID(bad)
if got == bad || !strings.HasPrefix(got, "req_") {
t.Errorf("%q became %q", bad, got)
}
}
if requestID("req_1") != "req_1" {
t.Error("a plain id was not kept")
}
}